from ipex_llm.transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True,
                                             load_in_low_bit="sym_int8", modules_to_not_convert=["vit"])
model = model.half()
model = model.eval()

import time
import torch

from ipex_llm.transformers import AutoModelForCausalLM
from transformers import AutoTokenizer

ckpt_path = "internlm-xcomposer2-vl-7b"
tokenizer = AutoTokenizer.from_pretrained(ckpt_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(ckpt_path, trust_remote_code=True,
                                             load_in_low_bit="sym_int8", modules_to_not_convert=["vit"])
model = model.half()
model = model.eval()

print(model)

query = '<ImageHere>Please describe this image in detail.'
image = 'image1.webp'

model = model.to('xpu')

with torch.inference_mode():
    for i in range(3):
        st = time.time()
        response, _ = model.chat(tokenizer, query=query, image=image, history=[], do_sample=False, max_new_tokens=1)
        et = time.time()
        print(response)
        print(et - st)